In [1]:

    

import sqlite3
conn = sqlite3.connect('pmcv1-full.db')
c = conn.cursor()


    

In [1]:

    

import graph_tool as gt


    

In [2]:

    

import cPickle as pickle


    

In [4]:

    

g = pickle.load(open("full_graph.p", "rb"))
pmid_vertex_dict = pickle.load(open("full_graph_pmid_vertex_dict.p", "rb"))


    

In [5]:

    

#plots
%matplotlib inline
import matplotlib.pyplot as plt
import numpy as np


    

In [6]:

    

#plot in degree
indegree = []
for vertex in g.vertices():
    indegree.append(vertex.in_degree())
plt.hist(indegree, np.arange(0,1000,20), log = True)


    

    
Out[6]:





(array([  7.55125200e+06,   1.43924000e+05,   3.12780000e+04,
          1.15570000e+04,   5.62600000e+03,   3.09700000e+03,
          1.96600000e+03,   1.29700000e+03,   8.93000000e+02,
          6.29000000e+02,   4.42000000e+02,   3.96000000e+02,
          2.73000000e+02,   2.54000000e+02,   1.74000000e+02,
          1.53000000e+02,   1.43000000e+02,   1.01000000e+02,
          7.70000000e+01,   8.10000000e+01,   6.30000000e+01,
          6.40000000e+01,   4.90000000e+01,   4.50000000e+01,
          3.80000000e+01,   3.80000000e+01,   3.10000000e+01,
          2.90000000e+01,   2.80000000e+01,   1.70000000e+01,
          2.20000000e+01,   1.50000000e+01,   1.40000000e+01,
          1.20000000e+01,   1.90000000e+01,   2.10000000e+01,
          1.00000000e+01,   1.10000000e+01,   1.20000000e+01,
          7.00000000e+00,   1.00000000e+01,   8.00000000e+00,
          8.00000000e+00,   9.00000000e+00,   1.10000000e+01,
          1.10000000e+01,   7.00000000e+00,   5.00000000e+00,
          3.00000000e+00]),
 array([  0,  20,  40,  60,  80, 100, 120, 140, 160, 180, 200, 220, 240,
        260, 280, 300, 320, 340, 360, 380, 400, 420, 440, 460, 480, 500,
        520, 540, 560, 580, 600, 620, 640, 660, 680, 700, 720, 740, 760,
        780, 800, 820, 840, 860, 880, 900, 920, 940, 960, 980]),
 <a list of 49 Patch objects>)






    

In [7]:

    

#plot out degree
outdegree = []
for vertex in g.vertices():
    outdegree.append(vertex.out_degree())
plt.hist(outdegree, np.arange(1,500,10), log = True)


    

    
Out[7]:





(array([  1.37930000e+05,   1.81997000e+05,   1.88491000e+05,
          1.52772000e+05,   1.05358000e+05,   6.18250000e+04,
          3.43840000e+04,   1.93530000e+04,   1.21570000e+04,
          8.02800000e+03,   5.55600000e+03,   4.07300000e+03,
          3.14300000e+03,   2.49200000e+03,   1.94900000e+03,
          1.54000000e+03,   1.18800000e+03,   9.34000000e+02,
          7.56000000e+02,   6.90000000e+02,   5.19000000e+02,
          3.97000000e+02,   3.38000000e+02,   2.68000000e+02,
          2.10000000e+02,   1.96000000e+02,   1.37000000e+02,
          1.32000000e+02,   8.90000000e+01,   8.60000000e+01,
          7.80000000e+01,   6.00000000e+01,   5.70000000e+01,
          3.30000000e+01,   3.70000000e+01,   4.20000000e+01,
          2.90000000e+01,   2.20000000e+01,   1.90000000e+01,
          2.00000000e+01,   9.00000000e+00,   1.30000000e+01,
          1.30000000e+01,   1.20000000e+01,   8.00000000e+00,
          8.00000000e+00,   6.00000000e+00,   6.00000000e+00,
          1.00000000e+01]),
 array([  1,  11,  21,  31,  41,  51,  61,  71,  81,  91, 101, 111, 121,
        131, 141, 151, 161, 171, 181, 191, 201, 211, 221, 231, 241, 251,
        261, 271, 281, 291, 301, 311, 321, 331, 341, 351, 361, 371, 381,
        391, 401, 411, 421, 431, 441, 451, 461, 471, 481, 491]),
 <a list of 49 Patch objects>)






    

In [8]:

    

# Let's plot its in-degree distribution
import graph_tool.stats as gtstats
in_hist = gtstats.vertex_hist(g, "in")

y = in_hist[0]
plt.figure(figsize=(10,6))
plt.errorbar(in_hist[1][:-1], in_hist[0], fmt="o", label="in")
plt.gca().set_yscale("log")
plt.gca().set_xlim(1, 1e3)
plt.gca().set_ylim(1.5,)
plt.xlabel("$k_{in}$")
plt.ylabel("$NP(k_{in})$")
plt.tight_layout()


    

In [11]:

    

#to save for web
# Let's plot its in-degree distribution
import graph_tool.stats as gtstats
in_hist = gtstats.vertex_hist(g, "in")

import matplotlib 
matplotlib.rc('xtick', labelsize=14) 
matplotlib.rc('ytick', labelsize=14)

y = in_hist[0]
plt.figure(figsize=(6,4))
plt.errorbar(in_hist[1][:-1], in_hist[0], fmt="o", label="in", alpha=0.5)
plt.gca().set_yscale("log")
plt.gca().set_xlim(1, 1e3)
plt.gca().set_ylim(1.5,)
plt.xlabel("Number of in-citations", fontsize=16)
plt.ylabel("Counts", fontsize=16)
plt.tight_layout()
plt.savefig('incites.svg')


    

In [7]:

    

# Let's plot its in-degree distribution
import graph_tool.stats as gtstats
in_hist = gtstats.vertex_hist(g, "out")

y = in_hist[0]
plt.figure(figsize=(10,6))
plt.errorbar(in_hist[1][:-1], in_hist[0], fmt="o", label="out")
plt.gca().set_yscale("log")
plt.gca().set_xlim(1, 600)
plt.gca().set_ylim(1.5,3e4)
plt.xlabel("$k_{in}$")
plt.ylabel("$NP(k_{in})$")
plt.tight_layout()


    

In [12]:

    

#to save for web
# Let's plot its in-degree distribution
import graph_tool.stats as gtstats
in_hist = gtstats.vertex_hist(g, "out")

import matplotlib 
matplotlib.rc('xtick', labelsize=14) 
matplotlib.rc('ytick', labelsize=14)

y = in_hist[0]
plt.figure(figsize=(6,4))
plt.errorbar(in_hist[1][:-1], in_hist[0], fmt="o", label="out", alpha = 0.5)
plt.gca().set_yscale("log")
plt.gca().set_xlim(1, 600)
plt.gca().set_ylim(1.5,3e4)
plt.xlabel("Number of out-citations", fontsize=16)
plt.ylabel("Counts", fontsize=16)
plt.tight_layout()
plt.savefig('outcites.svg')


    

In [28]:

    

g


    

    
Out[28]:





<Graph object, directed, with 7754378 vertices and 31231542 edges at 0x1154381d0>

In [59]:

    

rootpmid = 26247944
c.execute('''SELECT refpmid FROM refs WHERE pmid = ?''', [rootpmid])
output = c.fetchall()
print output


    

    




[(19464502,), (16130266,), (23897068,), (11719411,), (18241089,), (10739516,), (20502295,), (22441437,), (22495898,), (23362106,), (20664402,), (20166149,), (21386680,), (21520318,), (19777530,), (21257082,), (16374755,), (21439542,), (18817592,), (24150027,), (19516271,), (21474517,), (21185077,), (18991324,), (17322170,), (17071607,), (17368534,), (8405630,), (11844193,), (12420836,), (17899573,), (20232288,), (17149788,), (18332385,), (22753319,), (22846473,), (25136369,), (21115197,), (21295850,), (21254145,), (18480117,), (18259003,), (24752037,), (21389681,), (21210481,), (19656558,), (24970282,), (13680354,), (25653039,), (24959436,), (14764923,), (12618519,), (11943725,), (24166749,), (23331974,), (22906914,), (24444293,), (17553534,), (16957146,), (10077652,), (19215554,), (14678084,), (12648442,), (11932314,), (10875882,), (18771979,), (18655226,), (21252475,), (15280143,), (14710054,), (19497882,), (22036528,), (18259009,), (18753763,), (19035985,), (17618748,), (24607918,), (10849447,), (11095942,), (18264967,), (18978109,), (15994630,), (23980220,), (11040958,), (17227816,), (19436062,), (12000727,), (16647920,), (15257701,), (16102023,), (1372646,), (15118653,), (9065199,), (11239638,), (12624632,), (17365856,), (12049188,), (10471140,), (8572020,), (12422905,), (25280210,), (19879940,), (24185004,), (18166190,), (25028703,), (23300202,), (13956740,), (22544937,), (15343229,), (24103646,), (16877342,), (9931039,), (19602262,), (24852111,), (15867337,), (17322379,), (23526657,), (16224045,), (21093846,), (21079238,), (22438230,), (23830491,), (17346547,), (19285651,), (25981845,), (22364204,), (22251611,), (22852344,), (18218722,), (25346181,), (24356462,), (25122928,), (23890710,), (20598363,), (15572504,), (23280734,), (25530428,)]

In [51]:

    

for pmid in output:
    c.execute('''SELECT refpmid FROM refs WHERE pmid = ?''', [pmid[0]])
    print c.fetchall()


    

    




[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[(15733721,), (15947178,), (15743856,), (18259010,), (19541696,), (9725924,), (18480117,), (18259003,), (12796047,), (19087081,), (19369368,), (14662228,), (15760998,), (15924523,), (19516271,), (17443552,), (20687064,), (6668417,), (15094256,), (1393159,), (15856728,), (19155903,), (10925900,), (15387864,), (19477892,), (19336487,), (19502216,), (17556761,), (12193937,), (9396883,), (16098866,), (18475158,), (18725583,), (17903233,), (16169871,), (16157095,), (16678332,), (17975258,), (14520224,), (15042008,), (10521771,), (18819756,), (19586138,), (19273739,), (19813221,)]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[(19464502,), (18241089,), (11719411,), (12908998,), (20664402,), (22441437,), (22495898,), (22846473,), (22906914,), (24150027,), (19516271,), (6054057,), (3790464,), (1946241,), (11566964,), (12420836,), (11844193,), (17899573,), (16579806,), (22248988,), (15611362,), (9607693,), (10673728,), (15957992,), (7364175,), (7076094,), (3970084,), (2726114,), (8116715,), (8961572,), (9091016,), (9790376,), (9699746,), (10455476,), (11117771,), (10717813,), (10739512,), (11846705,), (12366481,), (14572364,), (15258539,), (20818956,), (18480117,), (24356462,), (7526035,), (10077652,), (12648442,), (14678084,), (19215554,), (11028579,), (11932314,), (12224070,), (15507981,), (16086443,), (17221926,), (17149788,), (12618519,), (15472115,), (14764923,), (17261644,), (11384693,), (11349200,), (11576575,), (12798535,), (15117604,), (14764795,), (18956425,), (18058842,), (18446652,), (18432600,), (22286035,), (19273739,), (21210481,), (24970282,), (15529289,), (22261192,), (21827221,), (24190934,), (23966056,), (24853452,), (24902880,), (21389681,)]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[(20482241,), (22018452,), (22267116,), (12908996,), (9494145,), (22102516,), (12618519,), (19850276,), (22000672,), (20232288,), (16098866,), (11932314,), (10875882,), (18771979,), (10940198,), (12850627,), (17299557,), (3105664,), (18058842,), (17212667,), (11932283,), (22542116,), (10967166,), (15743856,), (22647784,), (10077652,), (2662194,), (11844162,), (18332385,), (11197372,), (16202744,), (20150910,), (21867402,), (2200862,), (22986844,), (23557166,), (20818956,), (23929878,)]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[(15836870,), (15947178,), (15733721,), (17433431,), (17169180,), (17255351,), (18166303,), (8601931,), (12798543,), (17443552,), (17065044,), (16738957,), (17621696,), (16682385,), (12089065,), (15840747,), (12618519,), (2589440,), (18259009,), (19070896,), (18410657,), (12908998,), (13680354,), (8619929,), (8671397,), (9834463,), (8919031,), (11750869,), (10084962,), (9364601,), (9828203,), (15284729,), (15126581,), (15805796,), (9855593,), (14602804,), (16813756,), (17540809,), (17426082,), (15472115,), (15804781,), (14764923,), (12798535,), (14764795,), (11349200,), (11576575,), (18175241,), (17826405,), (16395263,), (15632339,), (16263826,), (17261644,), (18175243,), (1326540,), (7528549,), (12773481,), (15718503,), (16751767,), (18462791,), (17618745,), (16957146,), (17515455,), (17689640,), (18226617,), (18446652,), (15257701,), (16102023,), (1372646,), (15118653,), (9065199,), (11239638,), (12624632,), (17365856,), (12049188,), (12422905,), (8572020,), (10471140,), (17654753,), (11467696,), (17706337,), (9274585,), (15980640,), (15251949,), (18425486,), (12592259,), (11038466,), (12522563,), (12560345,), (18395872,), (16098864,), (9931039,), (12044339,), (15042003,), (18425484,), (15386446,), (15386451,), (11901066,), (16203989,), (17068167,), (17994635,), (17139698,), (9254906,), (14519395,), (16052496,), (9417060,), (16892462,), (15662668,), (14691998,), (17701664,), (17987603,), (15994630,), (18264967,), (18978109,), (6856484,), (11742106,), (15009185,), (10527825,), (19087972,), (18753763,), (17618748,), (17149788,), (15193867,), (19418482,), (18539259,), (16374755,), (17278173,), (12000727,), (1429570,), (7949102,), (16981714,), (18048494,), (16647920,), (16816069,), (10207505,), (11897673,), (10077652,), (11161967,), (15919757,), (17332520,), (17278174,), (15507981,), (15507982,), (11028579,), (11932314,), (17149788,), (18000943,), (17992705,), (14663836,), (12224070,), (6191321,), (13416279,), (15604363,), (17003355,), (16720654,), (17302880,), (16489932,), (18382866,), (17302880,), (8387282,), (10782362,), (12364390,), (16141406,), (18382699,), (14671210,), (10194466,), (18398593,), (16260641,), (15703421,), (18259044,), (19064815,), (10688503,), (18332385,), (18669736,), (18814185,), (18533121,), (16247621,), (12087083,), (16651850,), (16556680,), (15954879,), (17261642,), (14648175,), (18425489,), (12644709,), (15161720,), (15173234,), (12709362,), (14750173,), (18425492,), (18287768,), (17443845,), (16914840,), (17374602,), (18339795,), (18573524,), (18492697,)]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[]
[(9274585,), (11106320,), (15993701,), (15060106,), (20014413,), (22752761,), (9931039,), (11148193,), (12044339,), (15251945,), (15042003,), (16020496,), (17191257,), (23466432,), (23021026,), (19631923,), (22544937,), (22552373,), (12592259,), (16832835,), (19121662,), (19816026,), (22871389,), (12423866,), (16098864,), (21915884,), (22711432,), (22717986,), (22836687,), (23187089,), (11719487,), (14500595,), (10486150,)]

In [60]:

    

def addedge(graphobject, source, dest, vertexdict):
    if source not in vertexdict:
        v = graphobject.add_vertex()
        vertexdict[source] = int(v)
    if dest not in vertexdict:
        v = graphobject.add_vertex()
        vertexdict[dest] = int(v)
    graphobject.add_edge(vertexdict[source], vertexdict[dest])
    return graphobject, vertexdict


    

In [65]:

    

minigraph = gt.Graph()
rootpmid = 26247944
c.execute('''SELECT refpmid FROM refs WHERE pmid = ?''', [rootpmid])
children = c.fetchall()
vertexdict = dict()
for child in children:
    minigraph, vertexdict = addedge(minigraph, rootpmid, child[0], vertexdict)
    c.execute('''SELECT refpmid FROM refs WHERE pmid = ?''', [child[0]])
    secondchildren = c.fetchall()
    for child2 in secondchildren:
        minigraph, vertexdict = addedge(minigraph, child[0], child2[0], vertexdict)


    

In [78]:

    

minigraph


    

    
Out[78]:





<Graph object, directed, with 438 vertices and 531 edges at 0x116daa7d0>

In [212]:

    

import graph_tool.all as gt
import math


    

In [213]:

    

deg = minigraph.degree_property_map("in")
#deg.a = 4 * (math.sqrt(deg.a) * 0.5 + 0.4)
gt.graph_draw(minigraph, vertex_fill_color=deg)


    

    













    
Out[213]:





<PropertyMap object with key type 'Vertex' and value type 'vector<double>', for Graph 0x116daa7d0, at 0x116804090>

In [ ]:

    




    

In [ ]:

    

# CO-AUTHORSHIP NETWORK???
# https://graph-tool.skewed.de/static/doc/draw.html


    

In [ ]:

    




    

In [150]:

    

rev_pmid_vertex_dict = {v: k for k, v in pmid_vertex_dict.items()}


    

In [202]:

    

def addedge(graphobject, source, dest, vertexdict):
    if source not in vertexdict:
        v = graphobject.add_vertex()
        vertexdict[source] = int(v)
    if dest not in vertexdict:
        v = graphobject.add_vertex()
        vertexdict[dest] = int(v)
    graphobject.add_edge(vertexdict[source], vertexdict[dest])
    return graphobject, vertexdict


    

In [258]:

    

rootpmid = 26247944
minigraph2 = gt.Graph()
vertexdict2 = dict()
for neigh in g.vertex(pmid_vertex_dict[rootpmid]).out_neighbours(): 
    minigraph2, vertexdict2 = addedge(minigraph2, rootpmid, neigh, vertexdict2)
    try:
        for neigh2 in g.vertex(neigh).out_neighbours():
            minigraph2, vertexdict2 = addedge(minigraph2, neigh, neigh2, vertexdict2)
    except KeyError:
        print "{} not in graph g".format(neigh)


    

In [259]:

    

deg = minigraph2.degree_property_map("in")
gt.graph_draw(minigraph2, vertex_fill_color=deg)


    

    













    
Out[259]:





<PropertyMap object with key type 'Vertex' and value type 'vector<double>', for Graph 0x116ada850, at 0x117640950>

In [273]:

    

deg = minigraph2.degree_property_map("in")
deg.a = 2 * (np.sqrt(deg.a) * 0.5 + 0.4)
ebet = gt.betweenness(minigraph2)[1]
### for inline support
from IPython.display import SVG, display
###
gt.graphviz_draw(minigraph2, vcolor=deg, vorder=deg, elen=10, ecolor=ebet, 
                 eorder=ebet, output = "/tmp/3961631087.svg", output_format="svg",
                size = (25,25))
display(SVG(filename = "/tmp/3961631087.svg"))


    

In [214]:

    

pos = draw.arf_layout(minigraph2)
gt.graph_draw(minigraph2, pos=pos)


    

    













    
Out[214]:





<PropertyMap object with key type 'Vertex' and value type 'vector<double>', for Graph 0x1167a76d0, at 0x1185fc590>

In [215]:

    

pos = draw.sfdp_layout(minigraph2, cooling_step=0.95)
gt.graph_draw(minigraph2, pos=pos)


    

    













    
Out[215]:





<PropertyMap object with key type 'Vertex' and value type 'vector<double>', for Graph 0x1167a76d0, at 0x1185fca90>

In [196]:

    

pos = draw.arf_layout(minigraph2)
deg = minigraph2.degree_property_map("in")
gt.graph_draw(minigraph2, pos=pos, vertex_fill_color=deg)


    

    













    
Out[196]:





<PropertyMap object with key type 'Vertex' and value type 'vector<double>', for Graph 0x117703d50, at 0x116aae450>

In [219]:

    

state = gt.minimize_nested_blockmodel_dl(minigraph2, deg_corr=True)
gt.draw_hierarchy(state)


    

    




/usr/local/lib/python2.7/site-packages/graph_tool/draw/cairo_draw.py:619: UserWarning: Unknown parameter: layout_callback
  warnings.warn("Unknown parameter: " + k, UserWarning)
/usr/local/lib/python2.7/site-packages/graph_tool/draw/cairo_draw.py:619: UserWarning: Unknown parameter: key_press_callback
  warnings.warn("Unknown parameter: " + k, UserWarning)






    













    
Out[219]:





(<PropertyMap object with key type 'Vertex' and value type 'vector<double>', for Graph 0x1167a76d0, at 0x117640190>,
 <Graph object, directed, with 439 vertices and 438 edges at 0x117654e10>,
 <PropertyMap object with key type 'Vertex' and value type 'vector<double>', for Graph 0x117654e10, at 0x117648050>)

In [217]:

    

gt.graph_draw(minigraph2, pos=draw.sfdp_layout(minigraph2, cooling_step=0.99),
              vertex_fill_color=minigraph2.vertex_index, vertex_size=2,
              edge_pen_width=1)


    

    













    
Out[217]:





<PropertyMap object with key type 'Vertex' and value type 'vector<double>', for Graph 0x1167a76d0, at 0x1185fce50>

In [238]:

    

# LOOK INTO CO-AUTHORSHIP NETWORKS
# https://graph-tool.skewed.de/static/doc/draw.html
# http://www.ncbi.nlm.nih.gov/pmc/articles/PMC2721762/
g = minigraph2
g.purge_vertices()
state = gt.minimize_nested_blockmodel_dl(g, deg_corr=True)
t = gt.get_hierarchy_tree(state)[0]
tpos = pos = gt.radial_tree_layout(t, t.vertex(t.num_vertices() - 1), weighted=True)
cts = gt.get_hierarchy_control_points(g, t, tpos)
pos = g.own_property(tpos)
b = state.levels[0].b
shape = b.copy()
shape.a %= 14
gt.graph_draw(g, pos=pos, vertex_fill_color=b, vertex_shape=shape, edge_control_points=cts,
              edge_color=[0, 0, 0, 0.3], vertex_anchor=0)


    

    













    
Out[238]:





<PropertyMap object with key type 'Vertex' and value type 'vector<double>', for Graph 0x1167a76d0, at 0x11770ce50>

In [242]:

    

deg = minigraph2.degree_property_map("in")
deg.a = 4 * (np.sqrt(deg.a) * 0.5 + 0.4)
ebet = gt.betweenness(minigraph2)[1]
ebet.a /= ebet.a.max() / 10.
eorder = ebet.copy()
eorder.a *= -1
pos = gt.sfdp_layout(minigraph2)
control = g.new_edge_property("vector<double>")
for e in g.edges():
    d = np.sqrt(sum((pos[e.source()].a - pos[e.target()].a) ** 2)) / 5
    control[e] = [0.3, d, 0.7, d]
gt.graph_draw(minigraph2, pos=pos, vertex_size=deg, vertex_fill_color=deg, vorder=deg,
              edge_color=ebet, eorder=eorder, edge_pen_width=ebet,
              edge_control_points=control)
# SFDP force-directed layout of a Price network with 1500 nodes. The vertex size and color indicate the degree, 
# and the edge color and width the edge betweenness centrality.
# https://graph-tool.skewed.de/static/doc/draw.html


    

    













    
Out[242]:





<PropertyMap object with key type 'Vertex' and value type 'vector<double>', for Graph 0x1167a76d0, at 0x116adaa10>

In [218]:

    

gt.graph_draw(minigraph2, pos=draw.sfdp_layout(minigraph2, cooling_step=0.99),
              vertex_fill_color=minigraph2.vertex_index, vertex_size=2,
              edge_pen_width=1)


    

    













    
Out[218]:





<PropertyMap object with key type 'Vertex' and value type 'vector<double>', for Graph 0x1167a76d0, at 0x1185fce90>

In [ ]:

    




    

In [ ]:

    




    

In [245]:

    

rootpmid = 26247944
minigraph2 = gt.Graph()
vertexdict2 = dict()
for neigh in g.vertex(pmid_vertex_dict[rootpmid]).out_neighbours(): 
    minigraph2, vertexdict2 = addedge(minigraph2, rootpmid, neigh, vertexdict2)
    try:
        for neigh2 in g.vertex(neigh).out_neighbours():
            minigraph2, vertexdict2 = addedge(minigraph2, neigh, neigh2, vertexdict2)
            try:
                for neigh3 in g.vertex(neigh2).out_neighbours():
                    minigraph2, vertexdict2 = addedge(minigraph2, neigh2, neigh3, vertexdict2)
            except KeyError:
                    print "3rd degree node {} not in graph g".format(neigh)
    except KeyError:
        print "2nd degree node {} not in graph g".format(neigh)


    

In [247]:

    

deg = minigraph2.degree_property_map("in")
gt.graph_draw(minigraph2, vertex_fill_color=deg)


    

    













    
Out[247]:





<PropertyMap object with key type 'Vertex' and value type 'vector<double>', for Graph 0x116ada450, at 0x1167a7210>

In [402]:

    

def addedge(graphobject, source, dest, vertexdict):
    if source not in vertexdict:
        v = graphobject.add_vertex()
        vertexdict[source] = int(v)
    if dest not in vertexdict:
        v = graphobject.add_vertex()
        vertexdict[dest] = int(v)
    graphobject.add_edge(vertexdict[source], vertexdict[dest])
    return graphobject, vertexdict

#def addlayer(graphobject, rootnode, vertexdict, direction = 'out'):
#    if direction == 'out':
#        for neigh in graphobject.vertex(rootnode).out_neighbors():
#            graphobject, vertexdict = addedge(graphobject, rootnode, neigh, vertexdict)
#    elif direction == 'in':
#        for neigh in graphobject.vertex(rootnode).in_neighbors():
#            graphobject, vertexdict = addedge(graphobject, rootnode, neigh, vertexdict)
#    else:
#        print "Specify direction as either in or out"
       
import Queue
import graph_tool.all as gt
def buildlocalgraph(rootnode, mastergraph, indepth = 0, outdepth = 2):
    _g =gt.Graph()
    _vertexdict = dict()
    q = Queue.Queue()
    q.put((rootnode, 0))
    #first go in out direction
    while not q.empty():
        node = q.get()
        if node[1] <= outdepth:
            try:
                for neigh in _g.vertex(node[0]).out_neighbors():
                    _g, _vertexdict = addedge(_g, node[0], neigh, _vertexdict)
            except KeyError:
                "{} degree node {} not in graph g".format(node[1], node[0])
                
def buildlocalgraph(rootnode, mastergraph, indepth = 0, outdepth = 2):
    _g =gt.Graph()
    _vertexdict = dict()
    q = Queue.Queue()
    #first go in out direction
    q.put((rootnode, 0))
    while not q.empty():
        node = q.get()
        if node[1] < outdepth:
            try:
                for neigh in mastergraph.vertex(node[0]).out_neighbours():
                    _g, _vertexdict = addedge(_g, node[0], neigh, _vertexdict)
                    q.put((neigh, node[1]+1))
            except KeyError:
                "{} degree node {} not in graph g".format(node[1], node[0])
    #now go in in direction
    q.put((rootnode, 0))
    while not q.empty():
        node = q.get()
        if node[1] < indepth:
            try:
                for neigh in mastergraph.vertex(node[0]).in_neighbours():
                    _g, _vertexdict = addedge(_g, neigh, node[0], _vertexdict)
                    q.put((neigh, node[1]+1))
            except KeyError:
                "{} degree node {} not in graph g".format(node[1], node[0])
    return _g, _vertexdict


    

In [343]:

    

rootpmid = 26247944
minigraph3, vertexdict3 = buildlocalgraph(pmid_vertex_dict[rootpmid], g, 2, 0)


    

    




(5690856, 0)

In [383]:

    

def countoutneigh(g, node):
    count = 0
    for neigh in g.vertex(node).out_neighbours(): count+= 1
    return count

def countinneigh(g, node):
    count = 0
    for neigh in g.vertex(node).in_neighbours(): count+= 1
    return count


    

In [384]:

    

countoutneigh(g, 5690856)


    

    
Out[384]:





137

In [389]:

    

for neigh in g.vertex(5690856).out_neighbours():
    if countoutneigh(g, neigh) > 0: print countoutneigh(g, neigh), neigh


    

    




45 973846
82 1697637
38 2397896
196 2399401
33 5690874

In [393]:

    

countinneigh(g, 2399401)


    

    
Out[393]:





12

In [398]:

    

for i in range(2399400, 2399402):
    print countinneigh(g, i)


    

    




1
12

In [439]:

    

minigraph3, vertexdict3 = buildlocalgraph(2399401, g, 10, 10)


    

In [440]:

    

deg = minigraph3.degree_property_map("in")
gt.graph_draw(minigraph3, vertex_fill_color=deg, output_size=(1200,1200))


    

    













    
Out[440]:





<PropertyMap object with key type 'Vertex' and value type 'vector<double>', for Graph 0x117162ad0, at 0x10e7d3590>

In [418]:

    

minigraph3.purge_vertices()
state = gt.minimize_nested_blockmodel_dl(minigraph3, deg_corr=True)
t = gt.get_hierarchy_tree(state)[0]
tpos = pos = gt.radial_tree_layout(t, t.vertex(t.num_vertices() - 1), weighted=True)
cts = gt.get_hierarchy_control_points(minigraph3, t, tpos)
pos = minigraph3.own_property(tpos)
b = state.levels[0].b
shape = b.copy()
shape.a %= 14
gt.graph_draw(minigraph3, pos=pos, vertex_fill_color=b, vertex_shape=shape, edge_control_points=cts,
              edge_color=[0, 0, 0, 0.3], vertex_anchor=0)


    

    













    
Out[418]:





<PropertyMap object with key type 'Vertex' and value type 'vector<double>', for Graph 0x1185fcc10, at 0x11765f190>

In [421]:

    

minigraph3, vertexdict3 = buildlocalgraph(2399401, g, 10, 10)
state = gt.minimize_nested_blockmodel_dl(minigraph3, deg_corr=True)
gt.draw_hierarchy(state)


    

    













    
Out[421]:





(<PropertyMap object with key type 'Vertex' and value type 'vector<double>', for Graph 0x1172882d0, at 0x1173d0110>,
 <Graph object, directed, with 601 vertices and 600 edges at 0x117288d10>,
 <PropertyMap object with key type 'Vertex' and value type 'vector<double>', for Graph 0x117288d10, at 0x117288cd0>)

In [451]:

    

import math

g = minigraph3
print(g.num_vertices(), g.num_edges())

#reduce to only connected nodes
g = gt.GraphView(g,vfilt=lambda v: (v.out_degree() > 0) and (v.in_degree() > 0) )
g.purge_vertices()

print(g.num_vertices(), g.num_edges())

#use 1->Republican, 2->Democrat
red_blue_map = {1:(1,0,0,1),0:(0,0,1,1)}
plot_color = g.new_vertex_property('vector<double>')
g.vertex_properties['plot_color'] = plot_color
#for v in g.vertices():
#    plot_color[v] = red_blue_map[g.vertex_properties['value'][v]]  ######### USE THIS FOR GENDER??

#### THIS WAS USED FOR RED/BLUE POLITICAL AFIL
### SEE: http://stackoverflow.com/questions/238724/visualizing-undirected-graph-thats-too-large-for-graphviz

#edge colors
alpha=0.15
edge_color = g.new_edge_property('vector<double>')
g.edge_properties['edge_color']=edge_color
for e in g.edges():
    if plot_color[e.source()] != plot_color[e.target()]:
        if plot_color[e.source()] == (0,0,1,1):
            #orange on dem -> rep
            edge_color[e] = (255.0/255.0, 102/255.0, 0/255.0, alpha)
        else:
            edge_color[e] = (102.0/255.0, 51/255.0, 153/255.0, alpha)            
    #red on rep-rep edges
    elif plot_color[e.source()] == (1,0,0,1):
        edge_color[e] = (1,0,0, alpha)
    #blue on dem-dem edges
    else:
        edge_color[e] = (0,0,1, alpha)

state = gt.minimize_nested_blockmodel_dl(g, deg_corr=True)
bstack = state.get_bstack()
#t = gt.get_hierarchy_tree(bstack)[0]
tpos = pos = gt.radial_tree_layout(t, t.vertex(t.num_vertices() - 1), weighted=True)
cts = gt.get_hierarchy_control_points(g, t, tpos)
pos = g.own_property(tpos)
b = bstack[0].vp["b"]

#labels
text_rot = g.new_vertex_property('double')
g.vertex_properties['text_rot'] = text_rot
for v in g.vertices():
    if pos[v][0] >0:
        text_rot[v] = math.atan(pos[v][1]/pos[v][0])
    else:
        text_rot[v] = math.pi + math.atan(pos[v][1]/pos[v][0])

gt.graph_draw(g, pos=pos, vertex_fill_color=g.vertex_properties['plot_color'], 
            vertex_color=g.vertex_properties['plot_color'],
            edge_control_points=cts,
            vertex_size=10,
            #vertex_text=g.vertex_properties['label'],
            vertex_text_rotation=g.vertex_properties['text_rot'],
            vertex_text_position=1,
            vertex_font_size=9,
            edge_color=g.edge_properties['edge_color'],
            vertex_anchor=0,
            #bg_color=[0,0,0,1],
            output_size=[4024,4024])
            #output='polblogs_blockmodel.png')


    

    




(584, 966)
(152, 273)






    













    
Out[451]:





<PropertyMap object with key type 'Vertex' and value type 'vector<double>', for Graph 0x11728fc50, at 0x116653090>

In [452]:

    

import cPickle as pickle
g = pickle.load(open("full_graph.p", "rb"))
pmid_vertex_dict = pickle.load(open("full_graph_pmid_vertex_dict.p", "rb"))
rev_pmid_vertex_dict = {v: k for k, v in pmid_vertex_dict.items()}


    

In [461]:

    

for node in range(5000):
    inn = countinneigh(g,node)
    outn = countoutneigh(g, node)
    if (inn > 30) & (outn > 30): print node, rev_pmid_vertex_dict[node]


    

    




48 20502679
54 18803808
195 21042317
259 19783989
332 24304892
597 19874578
734 15933209
939 20804570
999 17211480
1000 16404366
1028 20964822
1042 19582160
1043 21884641
1130 21460848
1184 18349830
1214 22847613
1407 22821563
1514 20601955
1521 15535852
1880 16538219
1951 21917858
2059 18981049
2068 17088286
2218 17493263
2220 16643655
2232 20813035
2669 23700391
2696 23034130
2783 16948836
2798 19934210
3021 19621072
3223 20615901
3229 21423178
3230 19002258
3538 18682804
3705 23000897
3754 21247443
4603 21097890
4811 22006249
4820 16563223
4908 16751849

In [473]:

    

rootpmid = 20502679
minigraph3, vertexdict3 = buildlocalgraph(pmid_vertex_dict[rootpmid], g, 2, 5)
deg = minigraph3.degree_property_map("out") #out AND in how?
gt.graph_draw(minigraph3, vertex_fill_color=deg, output_size=(1200,1200))


    

    













    
Out[473]:





<PropertyMap object with key type 'Vertex' and value type 'vector<double>', for Graph 0x116d10410, at 0x116602350>

In [477]:

    

minigraph3, vertexdict3 = buildlocalgraph(pmid_vertex_dict[rootpmid], g, 5, 5)
minigraph3.purge_vertices()
state = gt.minimize_nested_blockmodel_dl(minigraph3, deg_corr=True)
t = gt.get_hierarchy_tree(state)[0]
tpos = pos = gt.radial_tree_layout(t, t.vertex(t.num_vertices() - 1), weighted=True)
cts = gt.get_hierarchy_control_points(minigraph3, t, tpos)
pos = minigraph3.own_property(tpos)
b = state.levels[0].b
shape = b.copy()
shape.a %= 14
gt.graph_draw(minigraph3, pos=pos, vertex_fill_color=b, vertex_shape=shape, edge_control_points=cts,
              edge_color=[0, 0, 0, 0.3], vertex_anchor=0)


    

    













    
Out[477]:





<PropertyMap object with key type 'Vertex' and value type 'vector<double>', for Graph 0x117410cd0, at 0x1164cc210>

In [476]:

    

minigraph3, vertexdict3 = buildlocalgraph(pmid_vertex_dict[rootpmid], g, 5, 5)
state = gt.minimize_nested_blockmodel_dl(minigraph3, deg_corr=True)
gt.draw_hierarchy(state)


    

    













    
Out[476]:





(<PropertyMap object with key type 'Vertex' and value type 'vector<double>', for Graph 0x117410650, at 0x1174a2c10>,
 <Graph object, directed, with 2319 vertices and 2318 edges at 0x116d10e50>,
 <PropertyMap object with key type 'Vertex' and value type 'vector<double>', for Graph 0x116d10e50, at 0x11653e050>)

In [2]:

    

def addedge(graphobject, source, dest, vertexdict, v_label):
    if source not in vertexdict:
        v = graphobject.add_vertex()
        vertexdict[source] = int(v)
        v_label[v] = str(rev_pmid_vertex_dict[dest])
    if dest not in vertexdict:
        v = graphobject.add_vertex()
        vertexdict[dest] = int(v)
        v_label[v] = str(rev_pmid_vertex_dict[dest])
    graphobject.add_edge(vertexdict[source], vertexdict[dest])
    return graphobject, vertexdict, v_label
       
import Queue
import graph_tool.all as gt
                
def buildlocalgraph(rootnode, mastergraph, indepth = 0, outdepth = 2):
    _g =gt.Graph()
    _vertexdict = dict()
    q = Queue.Queue()
    v_label = _g.new_vertex_property("string")
    #first go in out direction
    q.put((rootnode, 0))
    while not q.empty():
        node = q.get()
        if node[1] < outdepth:
            try:
                for neigh in mastergraph.vertex(node[0]).out_neighbours():
                    _g, _vertexdict, v_label = addedge(_g, node[0], neigh, _vertexdict, v_label)
                    q.put((neigh, node[1]+1))
            except KeyError:
                print "{} degree node {} not in graph g".format(node[1], node[0])
    #now go in in direction
    q.put((rootnode, 0))
    while not q.empty():
        node = q.get()
        if node[1] < indepth:
            try:
                for neigh in mastergraph.vertex(node[0]).in_neighbours():
                    _g, _vertexdict, v_label = addedge(_g, neigh, node[0], _vertexdict, v_label)
                    q.put((neigh, node[1]+1))
            except KeyError:
                "{} degree node {} not in graph g".format(node[1], node[0])
    return _g, _vertexdict, v_label


    

In [542]:

    

rootpmid = 20502679
#rev_pmid_vertex_dict = {v: k for k, v in pmid_vertex_dict.items()}
minigraph3, vertexdict3, v_label = buildlocalgraph(pmid_vertex_dict[rootpmid], g, 0, 2)
deg = minigraph3.degree_property_map("out") #out AND in how?
gt.graph_draw(minigraph3, vertex_fill_color=deg, vertex_text=v_label, output_size=(1200,1200))


    

    













    
Out[542]:





<PropertyMap object with key type 'Vertex' and value type 'vector<double>', for Graph 0x116aa7190, at 0x116ada250>

In [544]:

    

rootpmid = 20502679
#rev_pmid_vertex_dict = {v: k for k, v in pmid_vertex_dict.items()}
minigraph3, vertexdict3, v_label = buildlocalgraph(pmid_vertex_dict[rootpmid], g, 2, 2)
deg = minigraph3.degree_property_map("out") #out AND in how?
gt.graph_draw(minigraph3, vertex_fill_color=deg, output_size=(1200,1200))


    

    













    
Out[544]:





<PropertyMap object with key type 'Vertex' and value type 'vector<double>', for Graph 0x116aa7410, at 0x117162b90>

In [587]:

    

import collections
import nltk
import string
import nltk.stem.porter

def stem_tokens(tokens, stemmer):
    stemmed = []
    for item in tokens:
        stemmed.append(stemmer.stem(item))
    return stemmed

tokens = map(string.lower, nltk.word_tokenize(teststring))
stemmer = nltk.stem.porter.PorterStemmer()
stemmed = stem_tokens(tokens, stemmer)
count = collections.Counter(stemmed)
print count.most_common(100)


    

    




[(u',', 6), (u'we', 5), (u'.', 5), (u'the', 5), (u'in', 4), (u'it', 3), (u'and', 2), (u'then', 2), (u'text', 2), (u'tfidfvector', 2), (u'to', 2), (u"'s", 2), (u'a', 2), (u'word', 2), (u'remov', 2), (u'first', 2), (u'shakespear', 1), (u'term', 1), (u'creat', 1), (u'pass', 1), (u'an', 1), (u'own', 1), (u'through', 1), (u'tf-idf', 1), (u'file', 1), (u'our', 1), (u'articl', 1), (u'use', 1), (u'built', 1), (u"'", 1), (u'few', 1), (u'perform', 1), (u'next', 1), (u'call', 1), (u'which', 1), (u'input', 1), (u':', 1), (u'function', 1), (u'custom', 1), (u'rather', 1), (u'everi', 1), (u'that', 1), (u'each', 1), (u'stop', 1), (u"'known", 1), (u'initi', 1), (u'stem', 1), (u'lowercas', 1), (u'base', 1), (u'but', 1), (u'dictionari', 1), (u'particular', 1), (u'thing', 1), (u'given', 1), (u'than', 1), (u'on', 1), (u'convert', 1), (u'found', 1), (u'for', 1), (u'of', 1), (u'scikit-learn', 1), (u'punctuat', 1), (u'iter', 1), (u'doe', 1), (u'collect', 1), (u'token', 1), (u'calcul', 1), (u'fit_transform', 1), (u'nltk', 1)]

In [611]:

    

rootpmid = 20502679
c.execute('''SELECT abstract FROM abstracts WHERE pmid > ? AND pmid < ?''', [rootpmid-100, rootpmid+100])
corpus = c.fetchall()
print len(corpus)
#corpusflat = ['' + corpus[i] for i in range(len(corpus))]
for i, entry in enumerate(corpus):
    corpus[i] = str(corpus[i])


    

    




79

In [612]:

    

from sklearn.feature_extraction.text import TfidfVectorizer
tf = TfidfVectorizer(analyzer='word', ngram_range=(1,3), min_df = 0, stop_words = 'english')
tfidf_matrix =  tf.fit_transform(corpus)
feature_names = tf.get_feature_names() 
len(feature_names)